<!DOCTYPE html>
<html xmlns="http://www.w3.org/1999/xhtml">
  <head>

    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
    <meta content="Cask Data, Inc." name="author" />
<meta content="Copyright © 2016-2017 Cask Data, Inc." name="copyright" />


    <meta name="git_release" content="6.1.1">
    <meta name="git_hash" content="05fbac36f9f7aadeb44f5728cea35136dbc243e5">
    <meta name="git_timestamp" content="2020-02-09 08:22:47 +0800">
    <title>Creating Pipelines</title>

    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />
    <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/bootstrap-3.3.6/css/bootstrap-theme.min.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/bootstrap-sphinx.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-dynamicscrollspy-4.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/cdap-jquery.mCustomScrollbar.css" type="text/css" />
    <link rel="stylesheet" href="../_static/css/abixTreeList-2.css" type="text/css" />
    <link rel="stylesheet" href="../_static/cdap-bootstrap.css" type="text/css" />

    <script type="text/javascript">
      var DOCUMENTATION_OPTIONS = {
        URL_ROOT:    '',
        VERSION:     '6.1.1',
        COLLAPSE_INDEX: false,
        FILE_SUFFIX: '.html',
        HAS_SOURCE:  false
      };
    </script>
    <script type="text/javascript" src="../_static/jquery.js"></script>
    <script type="text/javascript" src="../_static/underscore.js"></script>
    <script type="text/javascript" src="../_static/doctools.js"></script>
    <script type="text/javascript" src="../_static/language_data.js"></script>

    <link rel="shortcut icon" href="../_static/favicon.ico"/>
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="top" title="Cask Data Application Platform 6.1.1 Documentation" href="../index.html" />
    <link rel="up" title="CDAP 数据流管道" href="index.html" />
    <link rel="next" title="Running Pipelines" href="running-pipelines.html" />
    <link rel="prev" title="CDAP 数据流设计器" href="studio.html" />
    <!-- block extrahead -->
    <meta charset='utf-8'>
    <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'>
    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1'>
    <meta name="apple-mobile-web-app-capable" content="yes">
    <!-- block extrahead end -->

</head>
<body role="document">

<!-- block navbar -->
<div id="navbar" class="navbar navbar-inverse navbar-default navbar-fixed-top">
    <div class="container-fluid">
      <div class="row">
        <div class="navbar-header">
          <!-- .btn-navbar is used as the toggle for collapsed navbar content -->
          <a class="navbar-brand" href="../table-of-contents/../../index.html">
            <span><img alt="CDAP logo" src="../_static/cdap_logo.svg"/></span>
          </a>

          <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".nav-collapse">
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
            <span class="icon-bar"></span>
          </button>

          <div class="pull-right">
            <div class="dropdown version-dropdown">
              <a href="#" class="dropdown-toggle" data-toggle="dropdown"
                role="button" aria-haspopup="true" aria-expanded="false">
                v 6.1.1 <span class="caret"></span>
              </a>
              <ul class="dropdown-menu">
                <li><a href="//docs.cdap.io/cdap/5.1.2/en/index.html">v 5.1.2</a></li>
                <li><a href="//docs.cdap.io/cdap/4.3.4/en/index.html">v 4.3.4</a></li>
              </ul>
            </div>
          </div>
          <form class="navbar-form navbar-right navbar-search" action="../search.html" method="get">
            <div class="form-group">
              <div class="navbar-search-image material-icons"></div>
              <input type="text" name="q" class="form-control" placeholder="  Search" />
            </div>
            <input type="hidden" name="check_keywords" value="yes" />
            <input type="hidden" name="area" value="default" />
          </form>

          <div class="collapse navbar-collapse nav-collapse navbar-right navbar-navigation">
            <ul class="nav navbar-nav"><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../index.html">简介</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link current" href="../table-of-contents/../../guides.html">手册</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../reference-manual/index.html">参考</a></li><li class="docsite-nav-tab-container"><a class="docsite-nav-tab-link " href="../table-of-contents/../../faqs/index.html">帮助</a></li>
            </ul>
          </div>

        </div>
      </div>
    </div>
  </div><!-- block navbar end -->
<!-- block main content -->
<div class="main-container container">
  <div class="row"><div class="col-md-2">
      <div id="sidebar" class="bs-sidenav scrollable-y-outside" role="complementary">
<!-- theme_manual: user-guide -->
<!-- theme_manual_highlight: guides -->
<!-- sidebar_title_link: ../table-of-contents/../../guides.html -->

  <div role="note" aria-label="manuals links"><h3><a href="../table-of-contents/../../guides.html">Guides</a></h3>

    <ul class="this-page-menu">
      <li class="toctree-l1"><b><a href="../table-of-contents/../../user-guide/index.html" rel="nofollow">用户手册</a></b>
      <nav class="pagenav">
      <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../index.html"> 简介</a></li>
<li class="toctree-l1"><a class="reference internal" href="../overview.html"> 概述</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/index.html"> 入门指南</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/campaign.html">MySQL 客户数据</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/nytimes-xml.html">纽约时报 XML 数据推送</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/stocks.html">股票选择</a></li>
<li class="toctree-l2"><a class="reference internal" href="../tutorials/fitbit.html">物联网 IoT 设备数据</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../data-preparation/index.html"> 数据预处理</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../data-preparation/concepts.html">      概念</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-preparation/directives/index.html">      数据处理指令</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/catalog-lookup.html">catalog-lookup</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/change-column-case.html">change-column-case</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/changing-case.html">changing-case</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/cleanse-column-names.html">cleanse-column-names</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/columns-replace.html">columns-replace</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/copy.html">copy</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/cut-character.html">cut-character</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/decode.html">decode</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/diff-date.html">diff-date</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/drop.html">drop</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/encode.html">encode</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/extract-regex-groups.html">extract-regex-groups</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/fail.html">fail</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/fill-null-or-empty.html">fill-null-or-empty</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/filter-row-if-matched.html">filter-row-if-matched</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/filter-row-if-true.html">filter-row-if-true</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/filter-rows-on.html">filter-rows-on</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/find-and-replace.html">find-and-replace</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/flatten.html">flatten</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/format-date.html">format-date</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/format-unix-timestamp.html">format-unix-timestamp</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/generate-uuid.html">generate-uuid</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/hash.html">hash</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/increment-variable.html">increment-variable</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/index-split.html">index-split</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/invoke-http.html">invoke-http</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/json-path.html">json-path</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/keep.html">keep</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/mask-number.html">mask-number</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/mask-shuffle.html">mask-shuffle</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/merge.html">merge</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-avro-file.html">parse-as-avro-file</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-avro.html">parse-as-avro</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-csv.html">parse-as-csv</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-date.html">parse-as-date</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-excel.html">parse-as-excel</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-fixed-length.html">parse-as-fixed-length</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-hl7.html">parse-as-hl7</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-json.html">parse-as-json</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-log.html">parse-as-log</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-simple-date.html">parse-as-simple-date</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-as-xml.html">parse-as-xml</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-timestamp.html">parse-timestamp</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/parse-xml-to-json.html">parse-xml-to-json</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/quantize.html">quantize</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/rename.html">rename</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/send-to-error.html">send-to-error</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/set-charset.html">set-charset</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/set-column.html">set-column</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/set-columns.html">set-columns</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/set-record-delim.html">set-record-delim</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/set-type.html">set-type</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/set-variable.html">set-variable</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/split-by-separator.html">split-by-separator</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/split-email.html">split-email</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/split-to-columns.html">split-to-columns</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/split-to-rows.html">split-to-rows</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/split-url.html">split-url</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/stemming.html">stemming</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/swap.html">swap</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/table-lookup.html">table-lookup</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/text-distance.html">text-distance</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/text-metric.html">text-metric</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/trim.html">trim</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/url-decode.html">url-decode</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/url-encode.html">url-encode</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/write-as-csv.html">write-as-csv</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/write-as-json-map.html">write-as-json-map</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/write-as-json-object.html">write-as-json-object</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/directives/xpath.html">xpath</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../data-preparation/functions/index.html">      函数</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/functions/json-functions.html">JSON 函数</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/functions/type-functions.html">类型函数</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/functions/geofence-functions.html">地理围栏函数</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/functions/dq-functions.html">数据质量函数</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/functions/date-functions.html">日期函数</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/functions/ddl-functions.html">DDL 函数</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../data-preparation/service/index.html">      服务</a><ul>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/service/admin.html">行政和管理服务</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/service/connection-properties.html">连接属性</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/service/connections.html">连接服务</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/service/execution.html">数据处理指令执行</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/service/request.html">请求格式规范</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/service/schema-registry.html">Schema 注册库</a></li>
<li class="toctree-l3"><a class="reference internal" href="../data-preparation/service/services.html">数据预处理服务</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../data-preparation/performance.html">性能</a></li>
<li class="toctree-l2"><a class="reference internal" href="../data-preparation/exclusion-and-aliasing.html">排除与别名</a></li>
</ul>
</li>
<li class="toctree-l1 current"><a class="reference internal" href="index.html"> 数据流管道</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="concepts-design.html"> 概念与设计</a></li>
<li class="toctree-l2"><a class="reference internal" href="getting-started.html"> 入门指南</a></li>
<li class="toctree-l2"><a class="reference internal" href="studio.html"> CDAP 数据流设计器</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#"> 创建数据流管道</a></li>
<li class="toctree-l2"><a class="reference internal" href="running-pipelines.html"> 运行数据流管道</a></li>
<li class="toctree-l2"><a class="reference internal" href="plugin-management.html"> 插件管理</a></li>
<li class="toctree-l2"><a class="reference internal" href="plugins/index.html"> 插件参考</a><ul>
<li class="toctree-l3"><a class="reference internal" href="plugins/actions/index.html"> Action Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/sources/index.html"> Source Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/transforms/index.html"> Transform Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/analytics/index.html"> Analytic Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/sinks/index.html"> Sink Plugins</a><ul class="simple">
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/shared-plugins/index.html"> Shared Plugins</a><ul>
<li class="toctree-l4"><a class="reference internal" href="plugins/shared-plugins/core.html">CoreValidator</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="plugins/post-run-plugins/index.html"> Post-run Plugins</a><ul class="simple">
</ul>
</li>
</ul>
</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../mmds/index.html"> 数据分析</a><ul>
<li class="toctree-l2"><a class="reference internal" href="../mmds/concepts.html"> Concepts</a></li>
<li class="toctree-l2"><a class="reference internal" href="../mmds/feature-gen.html"> Feature Generation</a></li>
<li class="toctree-l2"><a class="reference internal" href="../mmds/modeling.html"> Modeling</a></li>
<li class="toctree-l2"><a class="reference internal" href="../mmds/example.html"> Example</a></li>
</ul>
</li>
</ul>
</nav>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../developer-manual/index.html" rel="nofollow">开发手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../admin-manual/index.html" rel="nofollow">管理手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../integrations/index.html" rel="nofollow">集成手册</a>
      </li>
      <li class="toctree-l1"><a href="../table-of-contents/../../examples-manual/index.html" rel="nofollow">最佳实践</a>
      </li>
    </ul>
  </div></div>
    </div><div class="col-md-8 content" id="main-content">
    
  <div class="section" id="creating-pipelines">
<span id="user-guide-cdap-pipelines-creating-pipelines"></span><h1>Creating Pipelines<a class="headerlink" href="#creating-pipelines" title="Permalink to this headline">🔗</a></h1>
<p>Pipelines are created from <em>artifacts</em>. A number of artifacts are supplied with CDAP, and
custom artifacts can be created by developers. An artifact is a blueprint or template that—with the addition of a configuration file—is used to create an application.</p>
<p>A pipeline application is created by preparing a configuration that specifies the artifact
and which source, transformations (also known as transforms), and sinks are to be used to
create the application.</p>
<p>The sources, transformations, and sinks are packaged as extensions to CDAP known as
<strong>plugins</strong>, and can include actions to be taken at the start of pipeline run, at the end
of the run, and after the run has been completed. The plugins can be either those that are
packaged as part of CDAP or ones that have been installed separately.</p>
<p>The configuration can either be written as a JSON file or, in the case of the CDAP Studio,
specified in-memory.</p>
<p>CDAP currently provides two artifacts (referred to as <em>system artifacts</em>):</p>
<ul class="simple">
<li><code class="docutils literal notranslate"><span class="pre">cdap-data-pipeline</span></code> (for batch pipelines)</li>
<li><code class="docutils literal notranslate"><span class="pre">cdap-data-streams</span></code> (for real-time pipelines)</li>
</ul>
<p>which are used to create the different kinds of data pipeline applications.</p>
<p>Pipelines can be created using CDAP’s included visual editor (the <em>CDAP Studio</em>), using
command-line tools such the CDAP CLI and <code class="docutils literal notranslate"><span class="pre">curl</span></code>, or programmatically with scripts or
Java programs.</p>
<div class="section" id="methods-for-creating-pipelines">
<h2>Methods for Creating Pipelines<a class="headerlink" href="#methods-for-creating-pipelines" title="Permalink to this headline">🔗</a></h2>
<p>There are two different methods for creating pipelines:</p>
<ol class="arabic simple">
<li>Using the <em>CDAP Studio</em></li>
<li>Using command line tools (such as <code class="docutils literal notranslate"><span class="pre">curl</span></code>, the CDAP CLI, or the CDAP UI)</li>
</ol>
<p>Using the <strong>CDAP Studio</strong>, the basic operations are:</p>
<blockquote>
<div><ol class="arabic simple">
<li><strong>Create</strong> a new pipeline, either by starting from a <a class="reference internal" href="getting-started.html#cdap-pipelines-getting-started-cdap-studio"><span class="std std-ref">blank canvas</span></a>, starting from a
<a class="reference internal" href="#cdap-pipelines-creating-pipelines-pipeline-templates"><span class="std std-ref">template</span></a>, or by
<a class="reference internal" href="#cdap-pipelines-creating-pipelines-cloning"><span class="std std-ref">cloning</span></a> an already-published pipeline.</li>
<li><strong>Edit</strong> the pipeline in the CDAP Studio, setting appropriate configurations and
settings.</li>
<li><strong>Save</strong> the pipeline as you are working on it, as a draft pipeline, using a unique name.</li>
<li><strong>Validate</strong> the pipeline from within the CDAP Studio, to check that basic settings and
naming are correct.</li>
<li><strong>Publish</strong> the pipeline from within the CDAP Studio, which will translate the virtual
pipeline of the configuration into a physical pipeline with the specified name.</li>
</ol>
<p>At this point, the pipeline can be run.</p>
<p><strong>Note:</strong> Unlike many editors, the CDAP Studio does not allow draft pipelines to be
published “on top of” existing, published pipelines, as this would invalidate existing
logs, metrics, and datasets. Instead, it requires you to create a new name for any
newly-published pipelines.</p>
</div></blockquote>
<p>Using <strong>command line tools,</strong> the basic operations are:</p>
<blockquote>
<div><ol class="arabic simple">
<li><strong>Create</strong> a new pipeline by writing a configuration file, in JSON format following the
<span class="xref std std-ref">CDAP Pipeline configuration specification</span>,
either from an empty configuration, starting with an example or template, or re-using an
existing configuration file.</li>
<li><strong>Edit</strong> the JSON configuration file in an editor, setting appropriate configurations and
settings.</li>
<li><strong>Publish</strong> the pipeline either by using the Lifecycle RESTful API or CDAP CLI, which
will translate the virtual pipeline of the configuration file into a physical pipeline
with the specified name.</li>
</ol>
<p>Pipelines published using command line tools are visible within the CDAP UI, and
can be cloned and edited using the CDAP Studio.</p>
</div></blockquote>
</div>
<div class="section" id="batch-pipelines">
<h2>Batch Pipelines<a class="headerlink" href="#batch-pipelines" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="introduction">
<h3>Introduction<a class="headerlink" href="#introduction" title="Permalink to this headline">🔗</a></h3>
<p>Batch pipelines can be scheduled to run periodically using a cron expression, and can read
data from batch sources using either a MapReduce or Spark job. The batch application then
performs any (optional) transformations before writing to one or more batch sinks.</p>
<p>Action plugins can be added to perform special actions before the pipeline starts, and
when it successfully finishes. Post-run actions can be specified that will always run,
irregardless if the pipeline successfully completed.</p>
</div>
<div class="section" id="types-of-plugins">
<h3>Types of Plugins<a class="headerlink" href="#types-of-plugins" title="Permalink to this headline">🔗</a></h3>
<p>Batch pipelines, based on the <code class="docutils literal notranslate"><span class="pre">cdap-data-pipeline</span></code> application template, can include these plugins:</p>
<ul class="simple">
<li><a class="reference internal" href="plugins/actions/index.html#cdap-pipelines-plugins-actions"><span class="std std-ref">Actions</span></a></li>
<li><a class="reference internal" href="plugins/sources/index.html#cdap-pipelines-plugins-batch-sources"><span class="std std-ref">Batch Source Plugins</span></a></li>
<li><a class="reference internal" href="plugins/transforms/index.html#cdap-pipelines-plugins-transformations"><span class="std std-ref">Transformation Plugins</span></a></li>
<li><a class="reference internal" href="plugins/analytics/index.html#cdap-pipelines-plugins-analytics"><span class="std std-ref">Analytics Plugins</span></a></li>
<li><a class="reference internal" href="plugins/sinks/index.html#cdap-pipelines-plugins-batch-sinks"><span class="std std-ref">Batch Sink Plugins</span></a></li>
</ul>
</div>
<div class="section" id="how-does-it-work">
<h3>How Does It Work?<a class="headerlink" href="#how-does-it-work" title="Permalink to this headline">🔗</a></h3>
<p>The batch pipeline is created by taking a “virtual” pipeline (in the form of a
configuration file) and then creating a “physical” pipeline as a CDAP application with
appropriate CDAP programs to implement the configuration.</p>
<p>The programs used will depend on the engine chosen (MapReduce or Spark) and the plugins
used to build the pipeline. The available plugins are determined by those plugins that will
work with the <em>Data Pipeline</em> (the <code class="docutils literal notranslate"><span class="pre">cdap-data-pipeline</span></code> artifact), as listed above.</p>
</div>
<div class="section" id="building-a-pipeline">
<h3>Building a Pipeline<a class="headerlink" href="#building-a-pipeline" title="Permalink to this headline">🔗</a></h3>
<p>To create a batch pipeline, you can use either command line tools or the CDAP Studio.</p>
<p>To use the CDAP Studio to create a batch pipeline:</p>
<ul class="simple">
<li>Specify <em>Data Pipeline</em> (the <code class="docutils literal notranslate"><span class="pre">cdap-data-pipeline</span></code> artifact) as the application
template for your pipeline.</li>
<li>Click the icons in the left-sidebar to select the plugins you would like included in
your pipeline. In addition to the <a class="reference internal" href="plugins/actions/index.html#cdap-pipelines-plugins-actions"><span class="std std-ref">action plugins</span></a>,
the <a class="reference internal" href="plugins/transforms/index.html#cdap-pipelines-plugins-transformations"><span class="std std-ref">transform plugins</span></a>, and certain of
the <a class="reference internal" href="plugins/analytics/index.html#cdap-pipelines-plugins-analytics"><span class="std std-ref">analytics plugins</span></a>, you can use
any of the <a class="reference internal" href="plugins/sources/index.html#cdap-pipelines-plugins-batch-sources"><span class="std std-ref">batch source plugins</span></a> or the
<a class="reference internal" href="plugins/sinks/index.html#cdap-pipelines-plugins-batch-sinks"><span class="std std-ref">batch sink plugins</span></a>.</li>
<li>Typically, you will need at a minimum a source, a sink, and any optional transformations or analytics
that are required between the source and sink stages.</li>
<li>Action steps can be added before a source and after a sink. These will be run only at
the start (before a source) and only at the end if the pipeline successfully completes.</li>
<li>The <em>Settings</em> button allows you to specify the <em>Schedule</em>, “Post-run Actions* and <em>Engine</em> used
for the pipeline.</li>
<li>Specify a schedule for the batch pipeline, using either the <em>basic</em> or <em>advanced</em> specification.
The schedule uses the underlying operating system’s <code class="docutils literal notranslate"><span class="pre">cron</span></code> application.</li>
<li><a class="reference internal" href="plugins/post-run-plugins/index.html#cdap-pipelines-plugins-post-run-plugins"><span class="std std-ref">Post-run actions</span></a> can be specified, and
these will be run depending on the configuration; they can run even if the pipeline fails,
as they can be specified to run on one of <em>completion</em>, <em>success</em>, or <em>failure</em>. You can
have any number of post-run actions, and additional ones are added by clicking the <em>+</em>
button.</li>
<li>Specify an engine to use for the CDAP application. By default, <em>MapReduce</em> is used.</li>
<li>Complete all required information for each stage, and any optional information that your
particular use requires.</li>
<li>Save the pipeline under a unique name</li>
<li>Validate the pipeline, to check for errors.</li>
<li>Publish the pipeline, which will turn the virtual pipeline of the configuration file
into a physical pipeline of CDAP programs in a CDAP application.</li>
</ul>
<p>Note that publishing a pipeline can reveal errors that the validation step doesn’t catch, as
validation is not an exhaustive test.</p>
<p>At this point you can run your pipeline.</p>
<p>Details and an example of using command line tools to create a batch pipeline are in the
section on <span class="xref std std-ref">developing pipelines: creating a batch pipeline</span>.</p>
</div>
<div class="section" id="scheduling">
<h3>Scheduling<a class="headerlink" href="#scheduling" title="Permalink to this headline">🔗</a></h3>
<p>With the CDAP Studio, you can set a schedule for a batch pipeline that
will be used to run it. Note that as a schedule is set as part of the pipeline
configuration, a physical pipeline’s schedule cannot be altered except by creating a new
pipeline with a new schedule.</p>
<p>Two interfaces are available to create a schedule by time:</p>
<ul>
<li><p class="first">A <em>basic</em> interface, where you select the time increment (every minute, hour, day, week,
month, year) and the amount after the increment, as appropriate:</p>
<table border="1" class="docutils">
<colgroup>
<col width="6%" />
<col width="94%" />
</colgroup>
<tbody valign="top">
<tr class="row-odd"><td>Hour</td>
<td>Five-minute increment after the hour, 0 through 55 minutes</td>
</tr>
<tr class="row-even"><td>Day</td>
<td>Hour (twenty-four hour clock), plus five-minute increment after the hour, 0 through 55 minutes</td>
</tr>
<tr class="row-odd"><td>Week</td>
<td>Day of the week, plus hour (twenty-four hour clock), plus five-minute increment after the hour, 0 through 55 minutes</td>
</tr>
<tr class="row-even"><td>Month</td>
<td>Day of the month, plus hour (twenty-four hour clock), plus five-minute increment after the hour, 0 through 55 minutes</td>
</tr>
<tr class="row-odd"><td>Year</td>
<td>Date, plus hour (twenty-four hour clock), plus five-minute increment after the hour, 0 through 55 minutes</td>
</tr>
</tbody>
</table>
<p>If the specified time does not exist (for instance, you specified the 31st day of the
month, which doesn’t occur in June), the event is skipped until the next occurring event.</p>
<p>This <em>basic</em> schedule is converted into a <code class="docutils literal notranslate"><span class="pre">cron</span></code> expression for the configuration file.</p>
</li>
<li><p class="first">An <em>advanced</em> interface, which provides you access to the same interface as used in the
underlying <code class="docutils literal notranslate"><span class="pre">cron</span></code> program. The details of that program will depend on the operating
system used by the host of the CDAP Master process.</p>
</li>
</ul>
<p>With the CDAP Studio, you can also create a schedule that launches a pipeline when another pipeline is completed,
failed, or killed. Arguments and plugin properties from the triggering pipeline
can be passed to the triggered pipeline as arguments.</p>
<p><span class="xref std std-ref">HTTP request</span> can also be used to create such schedule.
Arguments and plugin properties from the triggering pipeline can be passed to the triggered pipeline by setting
a special schedule property. Set the ‘triggering.properties.mapping’ property to the stringified value of
the following JSON object:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
  <span class="s">&quot;arguments&quot;</span><span class="p">:</span> <span class="o">[</span>
    <span class="p">{</span>
      <span class="s">&quot;source&quot;</span><span class="p">:</span> <span class="o">&lt;</span><span class="n">runtime</span> <span class="n">argument</span> <span class="n">key</span> <span class="n">in</span> <span class="n">the</span> <span class="n">triggering</span> <span class="n">pipeline</span><span class="o">&gt;</span><span class="p">,</span>
      <span class="s">&quot;target&quot;</span><span class="p">:</span> <span class="o">&lt;</span><span class="n">runtime</span> <span class="n">argument</span> <span class="n">key</span> <span class="n">in</span> <span class="n">the</span> <span class="n">current</span> <span class="n">pipeline</span><span class="o">&gt;</span>
    <span class="p">},</span>
    <span class="p">...</span>
  <span class="o">]</span><span class="p">,</span>
  <span class="s">&quot;pluginProperties&quot;</span><span class="p">:</span> <span class="o">[</span>
    <span class="p">{</span>
      <span class="s">&quot;stageName&quot;</span><span class="p">:</span> <span class="o">&lt;</span><span class="n">stage</span> <span class="n">name</span> <span class="n">in</span> <span class="n">the</span> <span class="n">triggering</span> <span class="n">pipeline</span><span class="o">&gt;</span><span class="p">,</span>
      <span class="s">&quot;source&quot;</span><span class="p">:</span> <span class="o">&lt;</span><span class="n">plugin</span> <span class="n">property</span> <span class="n">key</span> <span class="n">in</span> <span class="n">the</span> <span class="n">given</span> <span class="n">stage</span> <span class="n">in</span> <span class="n">the</span> <span class="n">triggering</span> <span class="n">pipeline</span><span class="o">&gt;</span><span class="p">,</span>
      <span class="s">&quot;target&quot;</span><span class="p">:</span> <span class="o">&lt;</span><span class="n">runtime</span> <span class="n">argument</span> <span class="n">key</span> <span class="n">in</span> <span class="n">the</span> <span class="n">current</span> <span class="n">pipeline</span><span class="o">&gt;</span>
    <span class="p">},</span>
    <span class="p">...</span>
  <span class="o">]</span>
<span class="p">}</span>
</pre></div>
</div>
<p>For example, suppose you have pipeline ‘X’ in namespace ‘nsX’ and pipeline ‘Y’ in namespace ‘nsY’.
Pipeline ‘X’ is configured to write to a table set by the ‘output’ argument, and pipeline ‘Y’ is configured to read
from a table set by the ‘input’ argument. You want to create a schedule so that pipeline ‘Y’ is run whenever
pipeline ‘X’ completes, with pipeline ‘Y’ configured to read from the output table of pipeline ‘X’. In other words,
if pipeline ‘X’ completes with argument ‘output’=’xyTable’, you want pipeline ‘Y’ to run with argument
‘input’=’xyTable’. To do this, the ‘triggering.properties.mapping’ schedule property should contain
the ‘arguments’ field:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="s">&quot;arguments&quot;</span><span class="p">:</span> <span class="o">[</span>
  <span class="p">{</span>
    <span class="s">&quot;source&quot;</span><span class="p">:</span> <span class="s">&quot;output&quot;</span><span class="p">,</span>
    <span class="s">&quot;target&quot;</span><span class="p">:</span> <span class="s">&quot;input&quot;</span>
  <span class="p">}</span>
<span class="o">]</span>
</pre></div>
</div>
<p>In addition, you also want pipeline ‘Y’ to use the value set by the ‘row’ argument as the row key field of
the input table, and this row key field should be same as in the output table of pipeline ‘X’. In other words,
if the row key field defined in the plugin property ‘schema.row.field’ at stage ‘TableSink’ of pipeline ‘X’ is ‘id’,
you want pipeline ‘Y’ to run with argument ‘row’=’id’. To do this, the ‘triggering.properties.mapping’
schedule property should contain the ‘pluginProperties’ field:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="s">&quot;pluginProperties&quot;</span><span class="p">:</span> <span class="o">[</span>
  <span class="p">{</span>
    <span class="s">&quot;stageName&quot;</span><span class="p">:</span> <span class="s">&quot;TableSink&quot;</span>
    <span class="s">&quot;source&quot;</span><span class="p">:</span> <span class="s">&quot;schema.row.field&quot;</span><span class="p">,</span>
    <span class="s">&quot;target&quot;</span><span class="p">:</span> <span class="s">&quot;row&quot;</span>
  <span class="p">}</span>
<span class="o">]</span>
</pre></div>
</div>
<p>and the full HTTP PUT request body will be:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
  <span class="s">&quot;name&quot;</span><span class="p">:</span> <span class="s">&quot;SampleSchedule&quot;</span><span class="p">,</span>
  <span class="s">&quot;description&quot;</span><span class="p">:</span> <span class="s">&quot;A schedule that launches pipeline Y when pipeline X completes&quot;</span><span class="p">,</span>
  <span class="s">&quot;namespace&quot;</span><span class="p">:</span> <span class="s">&quot;nsY&quot;</span><span class="p">,</span>
  <span class="s">&quot;application&quot;</span><span class="p">:</span> <span class="s">&quot;Y&quot;</span><span class="p">,</span>
  <span class="s">&quot;applicationVersion&quot;</span><span class="p">:</span> <span class="s">&quot;-SNAPSHOT&quot;</span><span class="p">,</span>
  <span class="s">&quot;program&quot;</span><span class="p">:</span> <span class="p">{</span>
    <span class="s">&quot;programName&quot;</span><span class="p">:</span> <span class="s">&quot;SmartWorkflow&quot;</span><span class="p">,</span>
    <span class="s">&quot;programType&quot;</span><span class="p">:</span> <span class="s">&quot;WORKFLOW&quot;</span>
  <span class="p">},</span>
  <span class="s">&quot;properties&quot;</span><span class="p">:</span> <span class="p">{</span>
    <span class="s">&quot;triggering.properties.mapping&quot;</span><span class="p">:</span> <span class="s">&quot;{\&quot;arguments\&quot;:[{\&quot;source\&quot;:\&quot;output\&quot;,\&quot;target\&quot;:\&quot;input\&quot;}],\&quot;pluginProperties\&quot;:[{\&quot;stageName\&quot;:\&quot;TableSink\&quot;,\&quot;source\&quot;:\&quot;schema.row.field\&quot;,\&quot;target\&quot;:\&quot;row\&quot;}]}&quot;</span><span class="p">,</span>
    <span class="p">...</span>
  <span class="p">},</span>
  <span class="s">&quot;constraints&quot;</span><span class="p">:</span> <span class="o">[]</span><span class="p">,</span>
  <span class="s">&quot;trigger&quot;</span><span class="p">:</span> <span class="p">{</span>
    <span class="s">&quot;programId&quot;</span><span class="p">:</span> <span class="p">{</span>
        <span class="s">&quot;namespace&quot;</span><span class="p">:</span> <span class="s">&quot;nsX&quot;</span><span class="p">,</span>
        <span class="s">&quot;application&quot;</span><span class="p">:</span> <span class="s">&quot;X&quot;</span><span class="p">,</span>
        <span class="s">&quot;version&quot;</span><span class="p">:</span> <span class="s">&quot;-SNAPSHOT&quot;</span><span class="p">,</span>
        <span class="s">&quot;type&quot;</span><span class="p">:</span> <span class="s">&quot;WORKFLOW&quot;</span><span class="p">,</span>
        <span class="s">&quot;entity&quot;</span><span class="p">:</span> <span class="s">&quot;PROGRAM&quot;</span><span class="p">,</span>
        <span class="s">&quot;program&quot;</span><span class="p">:</span> <span class="s">&quot;SmartWorkflow&quot;</span>
    <span class="p">},</span>
    <span class="s">&quot;programStatuses&quot;</span><span class="p">:</span> <span class="o">[</span><span class="s">&quot;COMPLETED&quot;</span><span class="o">]</span><span class="p">,</span>
    <span class="s">&quot;type&quot;</span><span class="p">:</span> <span class="s">&quot;PROGRAM_STATUS&quot;</span>
  <span class="p">},</span>
  <span class="s">&quot;timeoutMillis&quot;</span><span class="p">:</span> <span class="mi">86400000</span>
<span class="p">}</span>
</pre></div>
</div>
</div>
<div class="section" id="engine">
<h3>Engine<a class="headerlink" href="#engine" title="Permalink to this headline">🔗</a></h3>
<p>You can specify the engine being used for a batch pipeline, either “MapReduce” (<code class="docutils literal notranslate"><span class="pre">mapreduce</span></code>)
or “Spark” (<code class="docutils literal notranslate"><span class="pre">spark</span></code>).</p>
<p>You set this either by selecting your choice using the <em>Settings</em> tool of the CDAP Studio,
or by setting the engine property in the configuration file for the pipeline:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="s">&quot;engine&quot;</span><span class="p">:</span> <span class="s">&quot;mapreduce&quot;</span><span class="p">,</span>
</pre></div>
</div>
<p>This determines the particular engine that will be used when the physical pipeline is
created.</p>
</div>
<div class="section" id="actions">
<span id="cdap-pipelines-creating-pipelines-actions"></span><h3>Actions<a class="headerlink" href="#actions" title="Permalink to this headline">🔗</a></h3>
<p>Actions can be configured for a batch pipeline, either by using the CDAP Studio or by
including a stage of type <code class="docutils literal notranslate"><span class="pre">action</span></code> in the configuration JSON file. The available actions
are determined by those available to the application template being used to create the
pipeline.</p>
<p>If configured, the action takes place either at the start or at the completion of a
pipeline run. All actions configured for the start will complete first before any other
stages, and all other stages will complete before any of the actions at the end are run.</p>
<p>Actions at the end will only run if the pipeline successfully completes. If you need an
action to run irregardless of completion, use a <a class="reference internal" href="#cdap-pipelines-creating-pipelines-post-run-actions"><span class="std std-ref">post-run action</span></a> instead.</p>
<p>Currently, action plugins are only available when using the <code class="docutils literal notranslate"><span class="pre">cdap-data-pipeline</span></code>
application template. Available action plugins are documented in the <a class="reference internal" href="plugins/actions/index.html#cdap-pipelines-plugins-actions"><span class="std std-ref">Plugin
Reference</span></a>, with this action available:</p>
<ul class="simple">
<li><em>SSH Action</em>, which establishes an SSH connection with a remote machine to execute a
command on that machine.</li>
</ul>
</div>
<div class="section" id="post-run-actions">
<span id="cdap-pipelines-creating-pipelines-post-run-actions"></span><h3>Post-run Actions<a class="headerlink" href="#post-run-actions" title="Permalink to this headline">🔗</a></h3>
<p>Post-run actions can be configured for a batch pipeline, either by using the CDAP Studio or
by setting the “postActions” property of the configuration JSON file. The available
actions are determined by the post-run plugins that are available to the application
template being used to create the pipeline.</p>
<p>If configured, the actions take place after the completion of a pipeline run,
and can happen depending of the status of the run. One of three conditions must be specified:</p>
<ul class="simple">
<li>completion (action takes place regardless of the status)</li>
<li>success (action takes place only upon success)</li>
<li>failure (action takes place only upon failure)</li>
</ul>
<p>Currently, post-run plugins are only available when using the <code class="docutils literal notranslate"><span class="pre">cdap-data-pipeline</span></code>
application template. Available post-run plugins are documented in the <a class="reference internal" href="plugins/post-run-plugins/index.html#cdap-pipelines-plugins-post-run-plugins"><span class="std std-ref">Plugin Reference</span></a>, with these actions currently available:</p>
<ul class="simple">
<li>sending an email</li>
<li>running a database query</li>
<li>making an HTTP request</li>
</ul>
</div>
</div>
<div class="section" id="real-time-pipelines">
<h2>Real-time Pipelines<a class="headerlink" href="#real-time-pipelines" title="Permalink to this headline">🔗</a></h2>
<div class="section" id="id1">
<h3>Introduction<a class="headerlink" href="#id1" title="Permalink to this headline">🔗</a></h3>
<p>Real-time pipelines are designed to generate micro batches of data at a regular interval, perform any
(optional) transformations and analytics, and then write to one or more sinks. As they are
intended to be run continuously, actions and post-run actions are not applicable or available.
Real-time pipelines do not operate on a record by record basis, but on a micro batch by micro batch basis.</p>
</div>
<div class="section" id="id2">
<h3>Types of Plugins<a class="headerlink" href="#id2" title="Permalink to this headline">🔗</a></h3>
<p>Real-time pipelines, based on the <code class="docutils literal notranslate"><span class="pre">cdap-data-streams</span></code> application template, can include these plugins:</p>
<ul class="simple">
<li><a class="reference internal" href="plugins/sources/index.html#cdap-pipelines-plugins-real-time-sources"><span class="std std-ref">Streaming Source Plugins</span></a></li>
<li><a class="reference internal" href="plugins/transforms/index.html#cdap-pipelines-plugins-transformations"><span class="std std-ref">Transformation Plugins</span></a></li>
<li><a class="reference internal" href="plugins/analytics/index.html#cdap-pipelines-plugins-analytics"><span class="std std-ref">Analytics Plugins</span></a></li>
<li><a class="reference internal" href="plugins/sinks/index.html#cdap-pipelines-plugins-batch-sinks"><span class="std std-ref">Batch Sink Plugins</span></a></li>
</ul>
<p>Despite the name, batch sink plugins are not limited to just batch pipelines.
The real-time pipeline artifact generates micro batches that can then be written to a batch sink.</p>
</div>
<div class="section" id="id3">
<h3>How Does It Work?<a class="headerlink" href="#id3" title="Permalink to this headline">🔗</a></h3>
<p>A real-time pipeline is created by taking a “virtual” pipeline (in the form of a
configuration file) and then creating a “physical” pipeline as a CDAP application with
appropriate CDAP programs to implement the configuration.</p>
<p>The application created will consist of a Spark Streaming program.</p>
</div>
<div class="section" id="id4">
<h3>Building a Pipeline<a class="headerlink" href="#id4" title="Permalink to this headline">🔗</a></h3>
<p>To create a real-time pipeline, you can use either the CDAP Studio or command line tools.</p>
<p>To use the CDAP Studio to create a real-time pipeline:</p>
<ul class="simple">
<li>Specify <em>Data Pipeline - Realtime</em> (the <code class="docutils literal notranslate"><span class="pre">cdap-data-streams</span></code> artifact) as the application
template for your pipeline.</li>
<li>Click the icons in the left-sidebar to select the plugins you would like included in
your pipeline. In addition to the <a class="reference internal" href="plugins/transforms/index.html#cdap-pipelines-plugins-transformations"><span class="std std-ref">transform plugins</span></a>
and certain of the <a class="reference internal" href="plugins/analytics/index.html#cdap-pipelines-plugins-analytics"><span class="std std-ref">analytics plugins</span></a>,
you can use any of the <a class="reference internal" href="plugins/sources/index.html#cdap-pipelines-plugins-real-time-sources"><span class="std std-ref">streaming source plugins</span></a> or the
<a class="reference internal" href="plugins/sinks/index.html#cdap-pipelines-plugins-batch-sinks"><span class="std std-ref">batch sink plugins</span></a>.</li>
<li>You will need at a minimum a source, a sink, and any optional transformations or analytics
that are needed between the source and sink stages.</li>
<li>The <em>Settings</em> button allows you to specify the batch interval for your pipeline. The batch interval controls
how often your sources will generate a micro batch of data. This must be a number followed
by a time unit, with ‘s’ for seconds, ‘m’ for minutes, and ‘h’ for hours.
For example, ’10s’ translates to ten seconds. This means the sources will generate a micro batch of data every
ten seconds.</li>
<li>Complete all required information for each stage, and any optional information that your
particular use requires.</li>
<li>Save the pipeline under a unique name</li>
<li>Validate the pipeline, to check for errors.</li>
<li>Publish the pipeline, which will turn the virtual pipeline of the configuration file
into a physical pipeline of CDAP programs in a CDAP application.</li>
</ul>
<p>Note that publishing a pipeline can reveal errors that the validation step doesn’t catch, as
validation is not an exhaustive test.</p>
<p>At this point you can run your pipeline.</p>
<p>Details and an example of using command line tools to create a real-time pipeline are in the
section on <span class="xref std std-ref">developing pipelines: creating a real-time pipeline</span>.</p>
</div>
</div>
<div class="section" id="field-configuration-settings">
<h2>Field Configuration Settings<a class="headerlink" href="#field-configuration-settings" title="Permalink to this headline">🔗</a></h2>
<p>These settings can be used in both batch and real-time pipelines.</p>
<div class="section" id="required-fields">
<h3>Required Fields<a class="headerlink" href="#required-fields" title="Permalink to this headline">🔗</a></h3>
<p>Certain fields are required to be configured in order for the plugin to work. These are
identified in the CDAP Studio configuration panel by a red dot, and are described in
the <a class="reference internal" href="plugins/index.html#cdap-pipelines-plugins"><span class="std std-ref">CDAP Pipelines Plugin Reference</span></a> documentation as <em>required</em>.</p>
</div>
</div>
<div class="section" id="field-macro-substitution">
<span id="cdap-pipelines-runtime-arguments-macros"></span><h2>Field Macro Substitution<a class="headerlink" href="#field-macro-substitution" title="Permalink to this headline">🔗</a></h2>
<p>Plugins can support macro-substitutable properties that allow placeholders for properties
that are unknown at configure time but are known and provided at runtime.</p>
<p>There are two types of macros supported in CDAP:</p>
<ul class="simple">
<li><a class="reference internal" href="#cdap-pipelines-macros-property-lookups"><span class="std std-ref">Property Lookups</span></a></li>
<li><a class="reference internal" href="#cdap-pipelines-macros-macro-functions"><span class="std std-ref">Macro Functions</span></a></li>
</ul>
<p>Fields that are macro-enabled are identified in the CDAP Studio UI and documented in
the <a class="reference internal" href="plugins/index.html#cdap-pipelines-plugins"><span class="std std-ref">CDAP Pipelines Plugin Reference</span></a>.</p>
<div class="section" id="property-lookups">
<span id="cdap-pipelines-macros-property-lookups"></span><h3>Property Lookups<a class="headerlink" href="#property-lookups" title="Permalink to this headline">🔗</a></h3>
<p><strong>Property lookups</strong> are specified using the syntax <code class="docutils literal notranslate"><span class="pre">${macro-name}</span></code>,
where <code class="docutils literal notranslate"><span class="pre">macro-name</span></code> is a key that is looked up in a hierarchy of sources.</p>
<p>The property lookup process for the key <code class="docutils literal notranslate"><span class="pre">${user-name}</span></code> follows these steps:</p>
<ol class="arabic simple">
<li>lookup in the <em>workflow token</em> for the key <code class="docutils literal notranslate"><span class="pre">user-name</span></code>, return value if found; else</li>
<li>lookup in the <em>runtime arguments</em> for the key <code class="docutils literal notranslate"><span class="pre">user-name</span></code>, return value if found; else</li>
<li>lookup in the <em>preference store</em> for the key <code class="docutils literal notranslate"><span class="pre">user-name</span></code>, return value if found; else</li>
<li>throw a “macro not found” exception.</li>
</ol>
<p>This order is used so that the most volatile source (the workflow token) takes precedence.</p>
<p>For instance, you might not know the name of a source stream until runtime. You could use
in the source stream’s <em>Stream Name</em> configuration:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">$</span><span class="p">{</span><span class="n">stream</span><span class="o">-</span><span class="n">name</span><span class="p">}</span>
</pre></div>
</div>
<p>and in the runtime arguments set a key-value pair such as:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">stream</span><span class="o">-</span><span class="n">name</span><span class="p">:</span> <span class="n">myDemoStream</span>
</pre></div>
</div>
<p>In the CDAP Pipelines UI, macro property look-ups are determined from the configuration and are set as runtime arguments.
After providing values for these arguments, the CDAP pipeline can be started:</p>
<blockquote>
<div><div class="figure align-center" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/stream-table-macro-runtime.png"><img alt="../_images/stream-table-macro-runtime.png" class="bordered-image" src="../_images/stream-table-macro-runtime.png" style="width: 6in;" /></a>
</div>
</div></blockquote>
<p><em>Notes:</em></p>
<ul class="simple">
<li>Information on setting preferences and runtime arguments is in the <span class="xref std std-ref">CDAP
管理手册, Preferences</span>. These can be set with the
<span class="xref std std-ref">Lifecycle</span> and <span class="xref std std-ref">Preferences</span> HTTP RESTful APIs.</li>
<li>To set values for macro keys through a CDAP pipeline’s preferences, see the
<span class="xref std std-ref">Preferences HTTP RESTful API</span>.</li>
</ul>
</div>
<div class="section" id="macro-functions">
<span id="cdap-pipelines-macros-macro-functions"></span><h3>Macro Functions<a class="headerlink" href="#macro-functions" title="Permalink to this headline">🔗</a></h3>
<p>In addition to property lookups, you can use predefined <strong>macro functions</strong>. Currently,
these functions are predefined and available:</p>
<ul class="simple">
<li><code class="docutils literal notranslate"><span class="pre">logicalStartTime</span></code></li>
<li><code class="docutils literal notranslate"><span class="pre">secure</span></code></li>
</ul>
<div class="section" id="logical-start-time-function">
<h4>Logical Start Time Function<a class="headerlink" href="#logical-start-time-function" title="Permalink to this headline">🔗</a></h4>
<p>The logicalStartTime macro function returns the logical start time of a run of the pipeline.</p>
<p>If no parameters are supplied, it returns the start time in milliseconds.
All parameters are optional. The function takes a time format, an offset, and a timezone as
arguments and uses the logical start time of a pipeline to perform the substitution:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">$</span><span class="p">{</span><span class="n">logicalStartTime</span><span class="p">(</span><span class="o">[</span><span class="n">timeFormat</span><span class="o">[</span><span class="p">,</span><span class="n">offset</span> <span class="o">[</span><span class="p">,</span><span class="n">timezone</span><span class="o">]</span><span class="p">)}</span>
</pre></div>
</div>
<p>where</p>
<table border="1" class="docutils">
<colgroup>
<col width="20%" />
<col width="80%" />
</colgroup>
<thead valign="bottom">
<tr class="row-odd"><th class="head">Parameter</th>
<th class="head">Description</th>
</tr>
</thead>
<tbody valign="top">
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">timeFormat</span></code> <em>(Optional)</em></td>
<td>Time format string, in the format of a Java <code class="docutils literal notranslate"><span class="pre">SimpleDateFormat</span></code></td>
</tr>
<tr class="row-odd"><td><code class="docutils literal notranslate"><span class="pre">offset</span></code> <em>(Optional)</em></td>
<td>Offset from the before the logical start time</td>
</tr>
<tr class="row-even"><td><code class="docutils literal notranslate"><span class="pre">timezone</span></code> <em>(Optional)</em></td>
<td>Timezone to be used for the logical start time</td>
</tr>
</tbody>
</table>
<p>For example, suppose the logical start time of a pipeline run is <code class="docutils literal notranslate"><span class="pre">2016-01-01T00:00:00</span></code> and
this macro is provided:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">$</span><span class="p">{</span><span class="n">logicalStartTime</span><span class="p">(</span><span class="n">yyyy</span><span class="o">-</span><span class="n">MM</span><span class="o">-</span><span class="n">dd</span><span class="sc">&#39;T&#39;</span><span class="n">HH</span><span class="o">-</span><span class="n">mm</span><span class="o">-</span><span class="n">ss</span><span class="p">,</span><span class="mf">1d</span><span class="o">-</span><span class="mi">4</span><span class="n">h</span><span class="o">+</span><span class="mi">30</span><span class="n">m</span><span class="p">)}</span>
</pre></div>
</div>
<p>The format is <code class="docutils literal notranslate"><span class="pre">yyyy-MM-dd'T'HH-mm-ss</span></code> and the offset is <code class="docutils literal notranslate"><span class="pre">1d-4h+30m</span></code> before the logical
start time. This means the macro will be replaced with <code class="docutils literal notranslate"><span class="pre">2015-12-31T03:30:00</span></code>, since the
offset translates to 20.5 hours. The entire macro evaluates to 20.5 hours before midnight
of January 1, 2016.</p>
</div>
<div class="section" id="secure-function">
<h4>Secure Function<a class="headerlink" href="#secure-function" title="Permalink to this headline">🔗</a></h4>
<p>The secure macro function takes in a single key as an argument and looks up the key’s
associated string value from the Secure Store. In order to perform the substitution, the
key provided as an argument must already exist in the secure store. This is useful for
performing a substitution with sensitive data.</p>
<p>For example, for a plugin that connects to a MySQL database, you could configure the
<em>password</em> property field with:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">$</span><span class="p">{</span><span class="n">secure</span><span class="p">(</span><span class="n">password</span><span class="p">)}</span>
</pre></div>
</div>
<p>which will pull the <em>password</em> from the Secure Store at runtime.</p>
</div>
</div>
<div class="section" id="recursive-macros">
<h3>Recursive Macros<a class="headerlink" href="#recursive-macros" title="Permalink to this headline">🔗</a></h3>
<p>Macros can be referential (refer to other macros), up to ten levels deep. For instance,
you might have a server that refers to a hostname and port, and supply these runtime
arguments, one of which is a definition of a macro that uses other macros:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="nl">hostname:</span> <span class="n">my</span><span class="o">-</span><span class="n">demo</span><span class="o">-</span><span class="n">host</span><span class="p">.</span><span class="na">example</span><span class="p">.</span><span class="na">com</span>
<span class="nl">port:</span> <span class="mi">9991</span>
<span class="n">server</span><span class="o">-</span><span class="n">address</span><span class="p">:</span> <span class="n">$</span><span class="p">{</span><span class="n">hostname</span><span class="p">}:</span><span class="n">$</span><span class="p">{</span><span class="n">port</span><span class="p">}</span>
</pre></div>
</div>
<p>In a pipeline configuration, you could use an expression such as:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">server</span><span class="o">-</span><span class="n">address</span><span class="p">:</span> <span class="n">$</span><span class="p">{</span><span class="n">server</span><span class="o">-</span><span class="n">address</span><span class="p">}</span>
</pre></div>
</div>
<p>expecting that it would be replaced with:</p>
<div class="highlight-java notranslate"><div class="highlight"><pre><span></span><span class="n">my</span><span class="o">-</span><span class="n">demo</span><span class="o">-</span><span class="n">host</span><span class="p">.</span><span class="na">example</span><span class="p">.</span><span class="na">com</span><span class="p">:</span><span class="mi">9991</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="validation">
<h2>Validation<a class="headerlink" href="#validation" title="Permalink to this headline">🔗</a></h2>
<p>From within the CDAP Studio, the validation button will examine the pipeline
to check that basic settings and naming are correct. Messages of any errors found will be
shown in the Studio. Note that this step is not exhaustive, and errors may still be found
when the pipeline is actually published.</p>
</div>
<div class="section" id="publishing">
<h2>Publishing<a class="headerlink" href="#publishing" title="Permalink to this headline">🔗</a></h2>
<p>Publishing a pipeline takes a <em>virtual</em> pipeline (such as a draft in the CDAP Application
Studio, or a configuration JSON file) and creates a <em>physical</em> pipeline (a CDAP
application) using the configuration file, plugin artifacts, and application template
artifacts.</p>
<p>Publishing can happen either from with the CDAP Studio or by using command
line tools, such as the <code class="docutils literal notranslate"><span class="pre">curl</span></code> command with the Lifecycle RESTful API, or the CDAP CLI
tool with its <code class="docutils literal notranslate"><span class="pre">create</span> <span class="pre">app</span></code> command.</p>
<p>Using either method, published pipelines are visible within the CDAP UI, and
can be cloned and edited using the CDAP Studio.</p>
</div>
<div class="section" id="templates-and-re-using-pipelines">
<h2>Templates and Re-using Pipelines<a class="headerlink" href="#templates-and-re-using-pipelines" title="Permalink to this headline">🔗</a></h2>
<p>Existing pipelines can be used to create new pipelines by:</p>
<ul class="simple">
<li>Using a <strong>pipeline template</strong></li>
<li><strong>Cloning</strong> an already-published pipeline and saving the resulting draft with a new name</li>
<li><strong>Exporting</strong> a configuration file, editing it, and then <strong>importing</strong> the revised file</li>
</ul>
<div class="section" id="pipeline-templates">
<span id="cdap-pipelines-creating-pipelines-pipeline-templates"></span><h3>Pipeline Templates<a class="headerlink" href="#pipeline-templates" title="Permalink to this headline">🔗</a></h3>
<p>A collection of predefined and preconfigured pipelines are available from within the CDAP Application
Studio through the controls at the top of the left side-bar. These templates can be used
as the starting point for either your own pipelines or your own pipeline templates.</p>
<div class="figure align-center" id="id5" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-studio-annotated.png"><img alt="../_images/cdap-studio-annotated.png" class="bordered-image" src="../_images/cdap-studio-annotated.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Studio:</strong> Annotations showing components</span></p>
</div>
<p>First, select which application template you wish to use, either <em>Data Pipeline</em> or
<em>ETL Real-time</em>.</p>
<p>Then, click <em>Template Gallery</em> to bring up a dialog that shows the available templates.
Click on the one you’d like to start with, and it will open, allowing you to begin customizing it
to your requirements.</p>
<p>These are the available templates:</p>
<ul class="simple">
<li><strong>Data Pipeline</strong><ul>
<li><strong>Model Trainer:</strong> Train model using Naive Bayes classifier</li>
<li><strong>Event Classifier:</strong> Classify events into spam or non-spam using a Naive Bayes model</li>
<li><strong>Log Data Aggregator:</strong> Aggregate log data by grouping IP and HTTP Status</li>
</ul>
</li>
<li><strong>ETL Real-time</strong><ul>
<li><strong>Kafka to HBase:</strong> Ingests in real time from Kafka into an HBase table</li>
<li><strong>Kafka to Stream:</strong> Ingests in real time from Kafka into a stream</li>
<li><strong>Kafka to OLAP Cube:</strong> Generate an OLAP Cube in real time from Kafka</li>
<li><strong>Twitter to HBase:</strong> Ingest real-time Twitter Stream into an HBase table</li>
<li><strong>Twitter to Stream:</strong> Ingest real-time Twitter Stream into a stream</li>
<li><strong>Amazon SQS to HBase:</strong> Real-time updates from Amazon Simple Queue Service into an HBase table</li>
</ul>
</li>
<li><strong>ETL Batch</strong> (deprecated as of CDAP 3.5.0; use <em>Data Pipeline</em> instead)<ul>
<li><strong>Stream to HBase:</strong> Periodically ingest from a stream into an HBase table</li>
</ul>
</li>
</ul>
</div>
<div class="section" id="cloning">
<span id="cdap-pipelines-creating-pipelines-cloning"></span><h3>Cloning<a class="headerlink" href="#cloning" title="Permalink to this headline">🔗</a></h3>
<p>Any existing pipeline that has been published, can be <em>cloned.</em> This creates an in-memory
copy of the pipeline with the same name and opens it within the CDAP Studio.</p>
<p>At this point, you can rename the pipeline to a unique name and then either save it as a
<a class="reference internal" href="studio.html#cdap-studio-pipeline-drafts"><span class="std std-ref">draft</span></a> or publish it as a new pipeline. As
you cannot save over an existing pipeline, all new pipelines need a unique name; a common
practice is to increment the names, from <em>Demo-1</em> to <em>Demo-2</em> with each new clone.</p>
</div>
<div class="section" id="exporting">
<h3>Exporting<a class="headerlink" href="#exporting" title="Permalink to this headline">🔗</a></h3>
<p>There are two ways you can export a pipeline configuration file:</p>
<ol class="arabic simple">
<li>From within the CDAP Studio; and</li>
<li>From within a CDAP pipeline configuration page.</li>
</ol>
<ol class="arabic">
<li><p class="first">From <strong>within the CDAP Studio</strong>, you can export a pipeline configuration JSON file using
the <em>Export…</em> button:</p>
<div class="figure align-center" id="id6" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-gs-1-5-buttons.png"><img alt="../_images/cdap-pipelines-gs-1-5-buttons.png" class="bordered-image" src="../_images/cdap-pipelines-gs-1-5-buttons.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Studio:</strong> Button labels, upper-right toolbar</span></p>
</div>
<p>Clicking the “Export…” button will bring up the export dialog:</p>
<div class="figure align-center" id="id7" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-studio-export.png"><img alt="../_images/cdap-studio-export.png" class="bordered-image" src="../_images/cdap-studio-export.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Studio:</strong> Export dialog, with display of configuration file</span></p>
</div>
<p>There are two similar actions you take. If you copy the text in the dialog and then
paste it into a text editor, you will have a JSON file that is the configuration of the
pipeline, but without the CDAP Studio UI information, such as the icon locations.</p>
<p>If you use the “Export” button, it will prompt for a file location before saving a
complete file with all the information required to recreate the pipeline in the CDAP Application
Studio, including details such as icon location. Otherwise, the two exports are
similar. The UI information is added in the <code class="docutils literal notranslate"><span class="pre">&quot;__ui__&quot;</span></code> object in the JSON configuration
file.</p>
</li>
<li><p class="first">From <strong>within a CDAP pipeline configuration</strong> page, there is an <em>Export</em> button:</p>
<div class="figure align-center" id="id8" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-detail-configuration.png"><img alt="../_images/cdap-pipelines-detail-configuration.png" class="bordered-image" src="../_images/cdap-pipelines-detail-configuration.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Pipelines:</strong> Configuration page, pipeline detail, showing <em>Export</em> button on right</span></p>
</div>
<p>Similar to exporting from with the CDAP Studio, exporting using the button will
produce a configuration with UI information, and copying the configuration visible
in the lower portion of the page will produce a configuration that does not include
the <code class="docutils literal notranslate"><span class="pre">&quot;__ui__&quot;</span></code> object in the JSON.</p>
</li>
</ol>
<p>Files created by exporting can be edited in a text editor and then imported to create new pipelines.</p>
</div>
<div class="section" id="importing">
<h3>Importing<a class="headerlink" href="#importing" title="Permalink to this headline">🔗</a></h3>
<p>From within the CDAP Studio, you can import a pipeline configuration JSON file to create a
new pipeline using the <em>Import Pipeline</em> button:</p>
<div class="figure align-center" id="id9" style="width: 100%">
<a class="bordered-image reference internal image-reference" href="../_images/cdap-pipelines-gs-1-5-buttons.png"><img alt="../_images/cdap-pipelines-gs-1-5-buttons.png" class="bordered-image" src="../_images/cdap-pipelines-gs-1-5-buttons.png" style="width: 6in;" /></a>
<p class="caption"><span class="caption-text"><strong>CDAP Studio:</strong> Button labels, upper-right toolbar</span></p>
</div>
<p>As determined by the configuration file, the application template will be set
appropriately, and may change from the current one.</p>
<p>A valid configuration file that meets the <span class="xref std std-ref">CDAP pipeline configuration file specification</span> is
required. It may be created from an existing pipeline by exporting its configuration file.</p>
</div>
</div>
</div>

</div>
    <div class="col-md-2">
      <div id="right-sidebar" class="bs-sidenav scrollable-y" role="complementary">
        <div id="localtoc-scrollspy">
        </div>
      </div>
    </div></div>
</div>
<!-- block main content end -->
<!-- block footer -->
<footer class="footer">
      <div class="container">
        <div class="row">
          <div class="col-md-2 footer-left"><a title="CDAP 数据流设计器" href="studio.html" />Previous</a></div>
          <div class="col-md-8 footer-center"><a class="footer-tab-link" href="../table-of-contents/../../reference-manual/licenses/index.html">Copyright</a> &copy; 2014-2020 Cask Data, Inc.&bull; <a class="footer-tab-link" href="//docs.cask.co/cdap/6.1.1/cdap-docs-6.1.1-web.zip" rel="nofollow">Download</a> an archive or
<a class="footer-tab-link" href="//docs.cask.co/cdap">switch the version</a> of the documentation
          </div>
          <div class="col-md-2 footer-right"><a title="Running Pipelines" href="running-pipelines.html" />Next</a></div>
        </div>
      </div>
    </footer>
<!-- block footer end -->
<script type="text/javascript" src="../_static/bootstrap-3.3.6/js/bootstrap.min.js"></script><script type="text/javascript" src="../_static/js/bootstrap-sphinx.js"></script><script type="text/javascript" src="../_static/js/abixTreeList-2.js"></script><script type="text/javascript" src="../_static/js/cdap-dynamicscrollspy-4.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script><script type="text/javascript" src="../_static/js/copy-to-clipboard.js"></script><script type="text/javascript" src="../_static/js/jquery.mousewheel.min.js"></script><script type="text/javascript" src="../_static/js/jquery.mCustomScrollbar.js"></script><script type="text/javascript" src="../_static/js/js.cookie.js"></script><script type="text/javascript" src="../_static/js/tabbed-parsed-literal-0.2.js"></script><script type="text/javascript" src="../_static/js/cdap-onload-javascript.js"></script><script type="text/javascript" src="../_static/js/cdap-version-menu.js"></script>
    <script src="https://cdap.gitee.io/docs/cdap/json-versions.js"/></script>
  </body>
</html>